{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression\n",
    "\n",
    "- Linear Data\n",
    "- Linear Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Import Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# visdom is a visualization tool from facebook\n",
    "\n",
    "from visdom import Visdom\n",
    "viz = Visdom()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Generate Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_data = 1000 \n",
    "num_epoch = 1000\n",
    "\n",
    "noise = init.normal(torch.FloatTensor(num_data,1),std=0.2)\n",
    "x = init.uniform(torch.Tensor(num_data,1),-10,10)\n",
    "y = 2*x+3\n",
    "y_noise = 2*(x+noise)+3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pane_352c4038cacf3e'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualize data with visdom\n",
    "\n",
    "input_data = torch.cat([x,y_noise],1)\n",
    "\n",
    "win=viz.scatter(\n",
    "    X = input_data,\n",
    "    opts=dict(\n",
    "        xtickmin=-10,\n",
    "        xtickmax=10,\n",
    "        xtickstep=1,\n",
    "        ytickmin=-20,\n",
    "        ytickmax=20,\n",
    "        ytickstep=1,\n",
    "        markersymbol='dot',\n",
    "        markersize=5,\n",
    "        markercolor=np.random.randint(0, 255, num_data),\n",
    "    ),\n",
    ")\n",
    "\n",
    "viz.updateTrace(\n",
    "    X = x,\n",
    "    Y = y,\n",
    "    win=win,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = nn.Linear(1,1)\n",
    "output = model(Variable(x))\n",
    "\n",
    "loss_func = nn.L1Loss()\n",
    "optimizer = optim.SGD(model.parameters(),lr=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 13.8858\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 11.3968\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 8.9434\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 6.6033\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 4.6256\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 3.4565\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 3.2108\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 3.0906\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.9810\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.8727\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.7645\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.6563\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.5482\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.4400\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.3318\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.2236\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.1154\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 2.0073\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.8991\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.7909\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.6871\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.5907\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.4980\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.4056\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.3133\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.2214\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.1298\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.0385\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.9479\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.8592\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.7730\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.6891\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.6130\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.5462\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.4896\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.4451\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.4116\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3871\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3696\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3572\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3480\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3418\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3374\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3340\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3316\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3302\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3293\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3287\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3283\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3280\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3278\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3276\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3275\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3275\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3274\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3274\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3274\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3274\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.3273\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "loss_arr =[]\n",
    "label = Variable(y_noise)\n",
    "for i in range(num_epoch):\n",
    "    output = model(Variable(x))\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    loss = loss_func(output,label)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if i % 10 == 0:\n",
    "        print(loss)\n",
    "    loss_arr.append(loss.data.numpy()[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Check Trained Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1.9998\n",
      "[torch.FloatTensor of size 1x1]\n",
      " \n",
      " 2.9834\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "param_list = list(model.parameters())\n",
    "print(param_list[0].data,param_list[1].data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Visualize output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pane_352c4038ee93b4'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "win_2=viz.scatter(\n",
    "    X = input_data,\n",
    "    opts=dict(\n",
    "        xtickmin=-10,\n",
    "        xtickmax=10,\n",
    "        xtickstep=1,\n",
    "        ytickmin=-20,\n",
    "        ytickmax=20,\n",
    "        ytickstep=1,\n",
    "        markersymbol='dot',\n",
    "        markercolor=np.random.randint(0, 255, num_data),\n",
    "        markersize=5,\n",
    "    ),\n",
    ")\n",
    "\n",
    "viz.updateTrace(\n",
    "    X = x,\n",
    "    Y = output.data,\n",
    "    win = win_2,\n",
    "    opts=dict(\n",
    "        xtickmin=-15,\n",
    "        xtickmax=10,\n",
    "        xtickstep=1,\n",
    "        ytickmin=-300,\n",
    "        ytickmax=200,\n",
    "        ytickstep=1,\n",
    "        markersymbol='dot',\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Visualize Loss Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = np.reshape([i for i in range(num_epoch)],newshape=[num_epoch,1])\n",
    "loss_data = np.reshape(loss_arr,newshape=[num_epoch,1])\n",
    "\n",
    "win2=viz.line(\n",
    "    X = x,\n",
    "    Y = loss_data,\n",
    "    opts=dict(\n",
    "        xtickmin=0,\n",
    "        xtickmax=num_epoch,\n",
    "        xtickstep=1,\n",
    "        ytickmin=0,\n",
    "        ytickmax=20,\n",
    "        ytickstep=1,\n",
    "        markercolor=np.random.randint(0, 255, num_epoch),\n",
    "    ),\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
